{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T02:43:59.075223Z",
     "iopub.status.busy": "2021-07-12T02:43:59.074678Z",
     "iopub.status.idle": "2021-07-12T02:43:59.316015Z",
     "shell.execute_reply": "2021-07-12T02:43:59.315303Z",
     "shell.execute_reply.started": "2021-07-12T02:43:59.075177Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T02:44:25.785010Z",
     "iopub.status.busy": "2021-07-12T02:44:25.784447Z",
     "iopub.status.idle": "2021-07-12T02:44:26.209677Z",
     "shell.execute_reply": "2021-07-12T02:44:26.208640Z",
     "shell.execute_reply.started": "2021-07-12T02:44:25.784965Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('./Datawhale_学术论文分类_数据集/train.csv', sep='\\t')\n",
    "test_df = pd.read_csv('./Datawhale_学术论文分类_数据集/test.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T02:45:22.194360Z",
     "iopub.status.busy": "2021-07-12T02:45:22.193813Z",
     "iopub.status.idle": "2021-07-12T02:45:22.209968Z",
     "shell.execute_reply": "2021-07-12T02:45:22.209417Z",
     "shell.execute_reply.started": "2021-07-12T02:45:22.194315Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paperid</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>categories</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train_00000</td>\n",
       "      <td>Hard but Robust, Easy but Sensitive: How Encod...</td>\n",
       "      <td>Neural machine translation (NMT) typically a...</td>\n",
       "      <td>cs.CL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train_00001</td>\n",
       "      <td>An Easy-to-use Real-world Multi-objective Opti...</td>\n",
       "      <td>Although synthetic test problems are widely ...</td>\n",
       "      <td>cs.NE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train_00002</td>\n",
       "      <td>Exploration of reproducibility issues in scien...</td>\n",
       "      <td>This is the first part of a small-scale expl...</td>\n",
       "      <td>cs.DL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train_00003</td>\n",
       "      <td>Scheduled Sampling for Transformers</td>\n",
       "      <td>Scheduled sampling is a technique for avoidi...</td>\n",
       "      <td>cs.CL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train_00004</td>\n",
       "      <td>Hybrid Forests for Left Ventricle Segmentation...</td>\n",
       "      <td>Machine learning models produce state-of-the...</td>\n",
       "      <td>cs.CV</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       paperid                                              title  \\\n",
       "0  train_00000  Hard but Robust, Easy but Sensitive: How Encod...   \n",
       "1  train_00001  An Easy-to-use Real-world Multi-objective Opti...   \n",
       "2  train_00002  Exploration of reproducibility issues in scien...   \n",
       "3  train_00003                Scheduled Sampling for Transformers   \n",
       "4  train_00004  Hybrid Forests for Left Ventricle Segmentation...   \n",
       "\n",
       "                                            abstract categories  \n",
       "0    Neural machine translation (NMT) typically a...      cs.CL  \n",
       "1    Although synthetic test problems are widely ...      cs.NE  \n",
       "2    This is the first part of a small-scale expl...      cs.DL  \n",
       "3    Scheduled sampling is a technique for avoidi...      cs.CL  \n",
       "4    Machine learning models produce state-of-the...      cs.CV  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T02:52:42.852019Z",
     "iopub.status.busy": "2021-07-12T02:52:42.851442Z",
     "iopub.status.idle": "2021-07-12T02:52:42.862286Z",
     "shell.execute_reply": "2021-07-12T02:52:42.861589Z",
     "shell.execute_reply.started": "2021-07-12T02:52:42.851971Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "tfidf = TfidfVectorizer(max_features=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T02:52:43.527233Z",
     "iopub.status.busy": "2021-07-12T02:52:43.526673Z",
     "iopub.status.idle": "2021-07-12T02:52:47.707478Z",
     "shell.execute_reply": "2021-07-12T02:52:47.707006Z",
     "shell.execute_reply.started": "2021-07-12T02:52:43.527187Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.49001398, 0.48456389, 0.48075577, 0.49429544, 0.48437187])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 只是用标题\n",
    "train_tfidf = tfidf.fit_transform(train_df['title'])\n",
    "\n",
    "clf = SGDClassifier()\n",
    "cross_val_score(clf, train_tfidf, train_df['categories'], cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T02:52:49.651835Z",
     "iopub.status.busy": "2021-07-12T02:52:49.651294Z",
     "iopub.status.idle": "2021-07-12T02:53:08.496802Z",
     "shell.execute_reply": "2021-07-12T02:53:08.496368Z",
     "shell.execute_reply.started": "2021-07-12T02:52:49.651788Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.61074496, 0.61574583, 0.60811756, 0.61068855, 0.61671008])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用摘要\n",
    "train_tfidf = tfidf.fit_transform(train_df['abstract'])\n",
    "\n",
    "clf = SGDClassifier()\n",
    "cross_val_score(clf, train_tfidf, train_df['categories'], cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T02:53:10.767173Z",
     "iopub.status.busy": "2021-07-12T02:53:10.766557Z",
     "iopub.status.idle": "2021-07-12T02:53:30.035371Z",
     "shell.execute_reply": "2021-07-12T02:53:30.034890Z",
     "shell.execute_reply.started": "2021-07-12T02:53:10.767125Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.62452566, 0.62793486, 0.62171349, 0.62580064, 0.63103586])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 标题和摘要\n",
    "train_df['title_abstract'] = train_df['title'] + ' ' + train_df['abstract']\n",
    "train_tfidf = tfidf.fit_transform(train_df['title_abstract'])\n",
    "\n",
    "clf = SGDClassifier()\n",
    "cross_val_score(clf, train_tfidf, train_df['categories'], cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T02:55:49.440915Z",
     "iopub.status.busy": "2021-07-12T02:55:49.440362Z",
     "iopub.status.idle": "2021-07-12T02:55:54.035370Z",
     "shell.execute_reply": "2021-07-12T02:55:54.034835Z",
     "shell.execute_reply.started": "2021-07-12T02:55:49.440869Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_df['title_abstract'] = test_df['title'] + ' ' + test_df['abstract']\n",
    "test_tfidf = tfidf.transform(test_df['title_abstract'])\n",
    "\n",
    "clf = SGDClassifier()\n",
    "clf.fit(train_tfidf, train_df['categories'])\n",
    "test_df['categories'] = clf.predict(test_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-12T02:56:17.165796Z",
     "iopub.status.busy": "2021-07-12T02:56:17.165224Z",
     "iopub.status.idle": "2021-07-12T02:56:17.185752Z",
     "shell.execute_reply": "2021-07-12T02:56:17.185136Z",
     "shell.execute_reply.started": "2021-07-12T02:56:17.165750Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_df[['paperid', 'categories']].to_csv('submit2.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
